# -*- coding: utf-8 -*-
# @Time    : 2021/9/27 14:46
# @rewrite : cl
import random
import os
import time
from selenium import webdriver
import requests
import pandas as pd
import base64
from Crypto.Cipher import AES
from tqdm import tqdm
from fake_useragent import UserAgent
import datetime
from dateutil.rrule import rrule, DAILY
from keywords import carkeywords, vehnames, brandnames, contentdict, branddict

# 加载基本信息
allkeywords = brandnames + carkeywords + vehnames

USER_AGENT_LIST = [
    'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36',
    'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.16 Safari/537.36',
    'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36',
    'Mozilla/5.0 (X11; CrOS i686 3912.101.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36',
    'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:17.0) Gecko/20100101 Firefox/17.0.6',
    'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1468.0 Safari/537.36',
    'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36',
    'Mozilla/5.0 (X11; CrOS i686 3912.101.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36',
    'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:90.0) Gecko/20100101 Firefox/90.0',
    'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:95.0) Gecko/20100101 Firefox/95.0',
    'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:96.0) Gecko/20100101 Firefox/96.0'
]
headers1 = {'accept': 'application/json, text/plain, */*',
            'accept-encoding': 'gzip, deflate, br', 'accept-language': 'zh-CN,zh;q=0.9', 'cache-control': 'no-cache',
            'content-length': '89', 'content-type': 'application/json;charset=UTF-8',
            'cookie': 'i18next=zh; MONITOR_WEB_ID=75a85826-c1b9-49f2-b7f8-7a1a383b4dd6; x-jupiter-uuid=16327104466868524; Hm_lvt_c36ebf0e0753eda09586ef4fb80ea125=1632709872,1632710446; Hm_lpvt_c36ebf0e0753eda09586ef4fb80ea125=1632710446; tt_scid=4hsZnfHNAtmWGSi4MWUNz-N1DPpijAYZKyzkeKneA2n.ECIhOKPqYLGHBiH8H5zm15f6; s_v_web_id=verify_ku21n812_2kf7njXo_yptW_4Q86_BXNX_Y67hNBvJscUD; _csrf_token=R1pNiIx5y40wvC5reN0CybZa',
            'origin': 'https://trendinsight.oceanengine.com', 'pragma': 'no-cache',
            'referer': 'https://trendinsight.oceanengine.com/arithmetic-index/analysis?keyword=WEY',
            'sec-ch-ua': '"Google Chrome";v="93", " Not;A Brand";v="99", "Chromium";v="93"', 'sec-ch-ua-mobile': '?0',
            'sec-ch-ua-platform': '"Windows"', 'sec-fetch-dest': 'empty', 'sec-fetch-mode': 'cors',
            'sec-fetch-site': 'same-origin', 'tea-uid': '6945252828761294374',
            'User-agent': 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36'}
index_toutiao_headers = {
    "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1",
    'Content-Type': 'application/json',

}
headers1['User-agent'] = str(UserAgent(verify_ssl=False).random)

# 定义一个便于获取 signature链接的浏览器
class Browser():
    def __init__(self, **kwargs, ):
        self.proxy = kwargs.get("proxy", None)
        self.api_url = kwargs.get("api_url", None)
        self.referrer = kwargs.get("referer", "https://trendinsight.oceanengine.com/")
        # TODO： update your executablePath
        self.executablePath = kwargs.get("executablePath", "/usr/bin/chromedriver")

        args = kwargs.get("browser_args", [])

        if len(args) == 0:
            self.args = []
        else:
            self.args = args
        options = webdriver.ChromeOptions()
        options.add_argument("--headless")
        options.add_argument("log-level=2")
        self.options = {
            "headless": True,
            "handleSIGINT": True,
            "handleSIGTERM": True,
            "handleSIGHUP": True,
        }
        if self.executablePath is not None:
            self.options["executablePath"] = self.executablePath

        self.browser = webdriver.Chrome(executable_path=self.executablePath, chrome_options=options)
        # self.browser =  webdriver.Chrome()
        self.browser.get('https://trendinsight.oceanengine.com/arithmetic-index')

    # 获指数signature链接
    def signature(self, keyword, start_date, end_date, app_name):
        sign_url = self.browser.execute_script('''
                var e={"url":"https://trendinsight.oceanengine.com/api/open/index/get_multi_keyword_hot_trend",
                        "method":"POST",
                        "data" : '{"keyword_list": ["%s"],"start_date": "%s","end_date": "%s","app_name": "%s"}'};
                var h = new XMLHttpRequest;h.open(e.method, e.url, true);
                h.setRequestHeader("accept","application/json, text/plain, */*");  
                h.setRequestHeader("content-type","application/json;charset=UTF-8");
                h.send(e.data);
                return h._url
        ''' % (keyword, start_date, end_date, app_name))
        return sign_url

    # 获取用户画像signature链接，
    def signature2(self, keyword, start_date, end_date, app_name):
        sign_url = self.browser.execute_script('''
                   var e={"url":"https://trendinsight.oceanengine.com/api/open/index/get_portrait",
                           "method":"POST",
                           "data" : '{"param":{"keyword":"%s","app_name":"%s","start_date":"%s","end_date":"%s"}}'};
                   var h = new XMLHttpRequest;h.open(e.method, e.url, true);
                   h.setRequestHeader("accept","application/json, text/plain, */*");  
                   h.setRequestHeader("content-type","application/json;charset=UTF-8");
                   h.send(e.data);
                   return h._url
           ''' % (keyword, app_name, start_date, end_date))
        return sign_url
    # 获取垂类榜单链接 按日取值
    def signature3(self, tag_id, end_date):
        sign_url = self.browser.execute_script('''
                   var e={"url":"https://trendinsight.oceanengine.com/api/open/content/content_creative_keywords",
                           "method":"POST",
                           "data" : '{"tag_id":"%s","end_date":"%s","period":"day"}'};
                   var h = new XMLHttpRequest;h.open(e.method, e.url, true);
                   h.setRequestHeader("accept","application/json, text/plain, */*");  
                   h.setRequestHeader("content-type","application/json;charset=UTF-8");
                   h.send(e.data);
                   return h._url
           ''' % (tag_id, end_date))
        return sign_url
    # 获取关联分析链接，
    def signature4(self, keyword, start_date, end_date, app_name):
        sign_url = self.browser.execute_script('''
                   var e={"url":"https://trendinsight.oceanengine.com/api/open/index/get_relation_word",
                           "method":"POST",
                           "data" : '{"param":{"keyword":"%s","start_date":"%s","end_date":"%s","app_name":"%s"}}'};
                   var h = new XMLHttpRequest;h.open(e.method, e.url, true);
                   h.setRequestHeader("accept","application/json, text/plain, */*");  
                   h.setRequestHeader("content-type","application/json;charset=UTF-8");
                   h.send(e.data);
                   return h._url
           ''' % (keyword, start_date, end_date, app_name))
        return sign_url

    #话题 还需调试
    def signature5(self, keyword, app_name):
        sign_url = self.browser.execute_script('''
                   var e={"url":"https://trendinsight.oceanengine.com/api/open/index/get_topic_suggest",
                           "method":"POST",
                           "data" : '{"keyword":"%s","app_name":"%s"}'};
                   var h = new XMLHttpRequest;h.open(e.method, e.url, true);
                   h.setRequestHeader("accept","application/json, text/plain, */*");  
                   h.setRequestHeader("content-type","application/json;charset=UTF-8");
                   h.send(e.data);
                   return h._url
           ''' % (keyword, app_name))
        return sign_url

    def signature6(self, topic_ids, begin_date, end_date, app_name):
        sign_url = self.browser.execute_script('''
                   var e={"url":"https://trendinsight.oceanengine.com/api/open/index/get_relation_word",
                           "method":"POST",
                           "data" : '{"app_name":"%s","topic_ids":["%s"],"end_date":"%s","begin_date":"%s"}'};
                   var h = new XMLHttpRequest;h.open(e.method, e.url, true);
                   h.setRequestHeader("accept","application/json, text/plain, */*");  
                   h.setRequestHeader("content-type","application/json;charset=UTF-8");
                   h.send(e.data);
                   return h._url
           ''' % (app_name, topic_ids, end_date, begin_date))
        return sign_url



    def close(self):
        self.browser.close()
        self.browser.quit()


browser = Browser()


# AES-128,秘钥 anN2bXA2NjYsamlh,ivamlheW91LHFpYW53
def decrtptlx(String):
    iv = "amlheW91LHFpYW53".encode(encoding='utf-8')
    key = 'anN2bXA2NjYsamlh'.encode(encoding='utf-8')
    cryptor = AES.new(key=key, mode=AES.MODE_CFB, IV=iv, segment_size=128)
    decode = base64.b64decode(String)
    plain_text = cryptor.decrypt(decode).decode('utf8')
    # plain_text = eval(plain_text)  #转化为字典
    # print(type(plain_texts), plain_texts)
    return plain_text
# 抖音\头条指数调用
def douyin_toutiao_index(keyword, start_date, end_date, app_name, headers=headers1):
    data = '{"keyword_list": ["%s"],"start_date": "%s","end_date": "%s","app_name": "%s"}' % (
        keyword, start_date, end_date, app_name)
    sign_url = browser.signature(keyword=keyword, start_date=start_date, end_date=end_date, app_name=app_name)
    print(sign_url)
    # 随机取headers
    try:
        string = requests.post(sign_url, headers=headers, data=data.encode()).json()['data']
        return string
    except:
        return None

# 用户画像调用 TODO：get_data2 should be revise，需要修改
def get_portrait(keyword, start_date, end_date, app_name):
    data = '{"param":{"keyword":"%s","app_name":"%s","start_date":"%s","end_date":"%s"}}' % (
        keyword, app_name, start_date, end_date)
    sign_url = browser.signature2(keyword=keyword, start_date=start_date, end_date=end_date, app_name=app_name)
    print(sign_url)
    # 修改以下语句以get data
    headers1[
        'User-agent'] = "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1"

    datastr = requests.post(sign_url, headers=headers1, data=data.encode(), timeout=10).json()['data']
    print('get datastr success:%s' % keyword)

    requests.Session.close()
    return datastr


# 垂类洞察调用
def content_creative_keywords(tag_id, end_date):
    data = '{"tag_id":"%s","end_date":"%s","period":"day"}' % (tag_id, end_date)
    sign_url = browser.signature3(tag_id=tag_id, end_date=end_date)
    #print(sign_url)
    str = requests.post(sign_url, headers=headers1, data=data.encode()).json()['data']
    requests.session().close()
    return str


#关联分析
def get_relation_words(keyword, start_date, end_date, app_name):
    data = '{"param":{"keyword":"%s","start_date":"%s","end_date":"%s","app_name":"%s"}}' % (
        keyword, start_date, end_date, app_name)
    sign_url = browser.signature4(keyword=keyword, start_date=start_date, end_date=end_date, app_name=app_name)

    headers1['User-agent'] = str(UserAgent(verify_ssl=False).random)
    datastr = requests.post(sign_url, headers=headers1, data=data.encode(), timeout=5).json()['data']
    print('datastr:',datastr[0:100])
    return datastr


#话题调用
def get_data5(keyword, app_name):
    data = '{"keyword":"%s","app_name":"%s"}' % (keyword, app_name)
    sign_url = browser.signature5(keyword, app_name)
    print(sign_url)
    datastr = requests.post(sign_url, headers=headers1, data=data.encode()).json()['data']
    return datastr


def get_data6(topic_ids, begin_date, end_date, app_name):
    data = '{"app_name":"%s","topic_ids":["%s"],"end_date":"%s","begin_date":"%s"}' % (
        app_name, topic_ids, end_date, begin_date)
    sign_url = browser.signature6(topic_ids=topic_ids, begin_date=begin_date, end_date=end_date, app_name=app_name)
    print(sign_url)
    datastr = requests.post(sign_url, headers=headers1, data=data.encode()).json()['data']
    return datastr





def writefile(listname):
    with open('successtimereport.txt', mode='rw', encoding='utf-8') as file1:
        file1.write(str(listname))
        print("'successtimereport.txt' 文件写入成功")


#获取当前日期第N天前
def getdate(beforeOfDay):
    today = datetime.datetime.now()
    # 计算偏移量
    offset = datetime.timedelta(days=-beforeOfDay)
    # 获取想要的日期的时间
    re_date = (today + offset).strftime('%Y%m%d')
    return re_date


if __name__ == "__main__":
    start_date = "20200101"  # 爬取最早时间
    end_date = '20210701'
    app_names = ['aweme', 'toutiao']

    browser.close()
